package crawler;

import cn.edu.hfut.dmic.webcollector.model.CrawlDatums;
import cn.edu.hfut.dmic.webcollector.model.Page;
import cn.edu.hfut.dmic.webcollector.plugin.berkeley.BreadthCrawler;
import util.JDBCHelper;

import org.jsoup.nodes.Document;
import org.springframework.jdbc.core.JdbcTemplate;

/**
 * Crawling news from hfut news
 *
 * @author hu
 */
public class NewsCrawler extends BreadthCrawler {
	JdbcTemplate jdbcTemplate = null;

	
	public void init(){
	
		try {
		    jdbcTemplate = JDBCHelper.createMysqlTemplate("mysql1",
		            "jdbc:mysql://localhost/newsdb?useUnicode=true&characterEncoding=utf8",
		            "root", "123456", 5, 30);

		    /*创建数据表*/
		    jdbcTemplate.execute("CREATE TABLE IF NOT EXISTS News ("
		            + "id int(11) NOT NULL AUTO_INCREMENT,"
		            + "title longtext,url varchar(200),html longtext,"
		            + "PRIMARY KEY (id)"
		            + ") ENGINE=MyISAM DEFAULT CHARSET=utf8;");
		    System.out.println("成功创建数据表 News");
		} catch (Exception ex) {
		    jdbcTemplate = null;
		    System.out.println("mysql未开启或JDBCHelper.createMysqlTemplate中参数配置不正确!");
		}
		
	}
	

    /**
     * @param crawlPath crawlPath is the path of the directory which maintains
     * information of this crawler
     * @param autoParse if autoParse is true,BreadthCrawler will auto extract
     * links which match regex rules from pag
     */
    public NewsCrawler(String crawlPath, boolean autoParse) {
    super(crawlPath, autoParse);
    /*start page*/
    this.addSeed("http://news.163.com");

    /*fetch url like http://news.hfut.edu.cn/show-xxxxxxhtml*/
    this.addRegex("http://news.163.com/*.*html");
    /*do not fetch jpg|png|gif*/
    this.addRegex("-.*\\.(jpg|png|gif).*");
    /*do not fetch url contains #*/
    this.addRegex("-.*#.*");
    
     init();
    
    }

    public void visit(Page page, CrawlDatums next) {
    	
    
    String url = page.getUrl();
    /*if page is news page*/
    if (page.matchUrl("http://news.163.com/*.*html")) {
        /*we use jsoup to parse page*/
        Document doc = page.getDoc();

        /*extract title and content of news by css selector*/
        String title = page.select("div[id=epContentLeft] > h1").first().text();
        String content = page.select("div[id=endText]").first().text();

        System.out.println("URL:\n" + url);
        System.out.println("title:\n" + title);
        System.out.println("content:\n" + content);
        
        if (jdbcTemplate != null) {
            int updates=jdbcTemplate.update("insert into News"
                +" (title,url,content) value(?,?,?)",
                    title, url, content);
            if(updates==1){
                System.out.println("mysql插入成功");
            }
        }

        /*If you want to add urls to crawl,add them to nextLink*/
        /*WebCollector automatically filters links that have been fetched before*/
        /*If autoParse is true and the link you add to nextLinks does not 
          match the regex rules,the link will also been filtered.*/
        //next.add("http://xxxxxx.com");
    }
    }

    public static void main(String[] args) throws Exception {
    NewsCrawler crawler = new NewsCrawler("crawl", true);
    crawler.setThreads(50);
    crawler.setTopN(100);
    //crawler.setResumable(true);
    /*start crawl with depth of 4*/
    crawler.start(4);
    }

}
